# -*- coding: UTF-8 -*-
__author__ = 'chzhu'

from bs4 import BeautifulSoup
from Logger import Logger
from datetime import datetime as DATETIME
import json

def strip_blanks(string):
    string = string.replace('\r', '').replace('\n', '').replace('\t', '')
    string = string.strip(' ')
    return string

class Parser(object):
    def __init__(self, html):
        self.html = html
        self.soup = BeautifulSoup(self.html)

        self.timelines = [] # store all timelines

        try:
            self.contents = self.refine_soup()
        except:
            raise Exception('refine soup fails!')

    def is_target(self, class_list):
        if class_list is None:
            return False
        class_type = ''
        for cls in class_list:
            class_type += cls + ' '
        class_type = class_type.strip(' ')
        if class_type.find('WB_cardwrap WB_feed_type S_bg2') != -1:
            return True
        else:
            return False

    def refine_soup(self):
        scripts = self.soup.find_all('script')
        for script in scripts:
            if script.text.find('feed_list_content') != -1:
                try: # process the script containing contents
                    contents = script.text
                    html = json.loads(contents[8:-1])['html']
                    soup = BeautifulSoup(html)
                    divs = soup.find_all('div')
                    tmlns = []
                    for div in divs:
                        try:
                            class_list = div['class']
                        except:
                            continue
                        if self.is_target(class_list):
                            tmlns.append(div)
                    return tmlns
                except Exception as e:
                    Logger.logger.debug(e)

        Logger.logger.debug('No feed list content found!')
        raise Exception('No feed list content found!')

    def parse_timelines(self):
       for tmln in self.contents:
            self.parse_timeline(tmln)
    def parse_timeline(self, timeline):
        dict = {}
        odict = {}

        try:
            dict['mid'] = self.parse_mid(timeline)
        except KeyError as ke:
            Logger.logger.debug(ke)
            exit(-1)

        try:
            odict['mid'] = self.parse_omid(timeline)
            if self.source_is_deleted(timeline):
                odict = {}
        except KeyError:
            pass


        dict['encrypt_mid'] = self.parse_encrypt_mid(timeline)
        if len(odict) != 0:
            odict['encrypt_mid'] = self.parse_encrypt_omid(timeline)

        try:
            dict['uid'] = self.parse_uid(timeline)
            if len(odict) != 0:
                odict['uid'] = self.parse_ouid(timeline)
        except:
            exit(-1)

        dict['retweet_num'] = self.parse_retweet_num(timeline)
        if len(odict) != 0:
            odict['retweet_num'] = self.parse_oretweet_num(timeline)

        dict['comment_num'] = self.parse_comment_num(timeline)
        if len(odict) != 0:
            odict['comment_num'] = self.parse_ocomment_num(timeline)

        dict['favourite_num'] = self.parse_favourite_num(timeline)
        if len(odict) != 0:
            odict['favourite_num'] = self.parse_ofavourite_num(timeline)

        dict['created_at'] = self.parse_created_at(timeline)
        if len(odict) != 0:
            odict['created_at'] = self.parse_ocreated_at(timeline)

        dict['from'] = self.parse_from(timeline)
        if len(odict) != 0:
            odict['from'] = self.parse_ofrom(timeline)

        dict['text'] = self.parse_text(timeline)
        if len(odict) != 0:
            odict['text'] = self.parse_otext(timeline)
            origin_user = self.parse_origin_user(timeline)
            dict['text'] += '//' + origin_user + ' ' + odict['text']

        dict['entity'] = self.parse_entity(timeline)
        if len(odict) != 0:
            odict['entity'] = self.parse_oentity(timeline)

        if len(odict) != 0:
            dict['source_mid'] = odict['mid']
            dict['source_uid'] = odict['uid']

        dict['mentions'] = self.parse_mentions(timeline)
        if len(odict) != 0:
            odict['mentions'] = self.parse_omentions(timeline)

        dict['check_in'] = self.parse_check_in(timeline)
        if len(odict) != 0:
            odict['check_in'] = self.parse_ocheck_in(timeline)

        dict['check_in_url'] = self.parse_check_in_url(timeline)
        if len(odict) != 0:
            odict['check_in_url'] = self.parse_ocheck_in_url(timeline)

        dict['is_deleted'] = '0'
        if len(odict) != 0:
            odict['is_deleted'] = '0'

        dict['timestamp'] = DATETIME.now()
        if len(odict) != 0:
            odict['timestamp'] = dict['timestamp']

        for key in dict:
            if dict[key] is None:
                dict[key] = ''
        self.timelines.append(dict)
        if len(odict) != 0:
            for key in odict:
                if odict[key] is None:
                    odict[key] = ''
            self.timelines.append(odict)

    def source_is_deleted(self, timeline):
        if timeline.find('div', 'empty_con clearfix') is None:
            return False
        else:
            return True

    def parse_check_in_url(self, timeline):
        try:
            card_links = timeline.find('div', {'node-type':'feed_list_content'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_place S_ficon'}) is not None:
                    check_in_url = cdlk['href']
                    return check_in_url
            return ''
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ocheck_in_url(self, timeline):
        try:
            card_links = timeline.find('div', {'node-type':'feed_list_reason'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_place S_ficon'}) is not None:
                    ocheck_in_url = cdlk['href']
                    return ocheck_in_url
            return ''
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_check_in(self, timeline):
        try:
            card_links = timeline.find('div', {'node-type':'feed_list_content'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_place S_ficon'}) is not None:
                    check_in = cdlk.find('em', 'W_autocut S_link1').text
                    return check_in
            return ''
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ocheck_in(self, timeline):
        try:
            card_links = timeline.find('div', {'node-type':'feed_list_reason'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_place S_ficon'}) is not None:
                    ocheck_in = cdlk.find('em', 'W_autocut S_link1').text
                    return ocheck_in
            return ''
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_mentions(self, timeline):
        try:
            text = timeline.find('div', {'node-type':'feed_list_content'})
            mention_list = text.find_all('a', {'extra-data':'type=atname'})
            mentions = ''
            for mention in mention_list:
                mentions += mention.text + ', '
            return mentions.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_omentions(self, timeline):
        try:
            otext = timeline.find('div', {'node-type':'feed_list_reason'})
            omention_list = otext.find_all('a', {'extra-data':'type=atname'})
            omentions = ''
            for omention in omention_list:
                omentions += omention.text + ', '
            return omentions.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_entity(self, timeline):
        entity = {
            'img':'',
            'video':'',
            'audio':'',
            'link':'',
            'event':''
        }
        try:
            media_box = timeline.find('div', {'class':'WB_media_wrap clearfix', 'node-type':'feed_list_media_prev'})
            card_links = timeline.find('div', {'node-type':'feed_list_content'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})

            entity['img'] = self.parse_img(media_box)
            entity['video'] = self.parse_video(card_links)
            entity['audio'] = self.parse_audio(card_links)
            entity['link'] = self.parse_link(card_links)
            entity['event'] = self.parse_event(card_links)
            entity['product'] = self.parse_product(card_links)
            entity['other'] = self.parse_other(card_links)

            entities = ''
            for ent in entity:
                if entity[ent] != '':
                    entities += ent + '::' + entity[ent] + '; '
            return entities.strip('; ')

        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_img(self, media_box):
        if media_box is None:
            return ''
        try:
            imgs = media_box.find_all('img')
            entity_img = ''
            for img in imgs:
                entity_img += img['src'] + ', '
            return entity_img.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_video(self, card_links):
        video = ''
        try:
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_video S_ficon'}) is not None:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    video += title + ':' + cdlk['href'] + ', '
            return video.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_audio(self, card_links):
        audio = ''
        try:
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_music S_ficon'}) is not None:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    audio += title + ':' + cdlk['href'] + ', '
            return audio.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_link(self, card_links):
        link = ''
        try:
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_link S_ficon'}) is not None:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    link += title + ':' + cdlk['href'] + ', '
            return link.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_event(self, card_links):
        event = ''
        try:
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_event S_ficon'}) is not None:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    event += title + ':' + cdlk['href'] + ', '
            return event.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_product(self, card_links):
        product = ''
        try:
            for cdlk in card_links:
                if cdlk.find('i', {'class':'W_ficon ficon_cd_product S_ficon'}) is not None:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    product += title + ':' + cdlk['href'] + ', '
            return product.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_other(self, card_links):
        categories = [
            'W_ficon ficon_cd_video S_ficon',
            'W_ficon ficon_cd_music S_ficon',
            'W_ficon ficon_cd_link S_ficon',
            'W_ficon ficon_cd_event S_ficon',
            'W_ficon ficon_cd_product S_ficon',
            'W_ficon ficon_cd_place S_ficon'
        ]

        other = ''
        try:
            for cdlk in card_links:
                is_categoried = False
                for cat in categories:
                    if cdlk.find('i', {'class':cat}) is not None:
                        is_categoried = True
                        break
                if is_categoried is False:
                    title = cdlk.find('em', {'class':'W_autocut S_link1'}).text
                    title = strip_blanks(title)
                    other += title + ':' + cdlk['href'] + ', '
                else:
                    continue
            return other.strip(', ')
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_oentity(self, timeline):
        oentity = {
            'img':'',
            'video':'',
            'audio':'',
            'link':'',
            'event':''
        }
        try:
            media_box = timeline.find('div', {'node-type':'feed_list_media_prev'})
            card_links = timeline.find('div', {'node-type':'feed_list_reason'}).find_all('a', {'class':'W_btn_b btn_22px W_btn_cardlink'})

            oentity['img'] = self.parse_img(media_box)
            oentity['video'] = self.parse_video(card_links)
            oentity['audio'] = self.parse_audio(card_links)
            oentity['link'] = self.parse_link(card_links)
            oentity['event'] = self.parse_event(card_links)
            oentity['product'] = self.parse_product(card_links)
            oentity['other'] = self.parse_other(card_links)

            oentities = ''
            for oent in oentity:
                if oentity[oent] != '':
                    oentities += oent + '::' + oentity[oent] + '; '
            return oentities.strip('; ')
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_text(self, timeline):
        try:
            text = timeline.find('div', {'node-type':'feed_list_content'})
            text = strip_blanks(text.text)
            return text
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_otext(self, timeline):
        try:
            otext = timeline.find('div', {'node-type':'feed_list_reason'})
            otext = strip_blanks(otext.text)
            return otext
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_origin_user(self, timeline):
        try:
            origin_user = timeline.find('a', {'node-type':'feed_list_originNick'})
            origin_user = strip_blanks(origin_user.text)
            return origin_user
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_from(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            app_source = date_bar[-1]
            app_source = app_source.find('a', {'action-type':'app_source'})
            if app_source is None: # some app_source labels are not marked as app_source
                app_source = date_bar[-1].find('a', {'node-type':'feed_list_item_date'}).find_next_sibling("a")
            app_source = app_source.text
            app_source = strip_blanks(app_source)
            return app_source
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ofrom(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            oapp_source = date_bar[0]
            oapp_source = oapp_source.find('a', {'action-type':'app_source'})
            if oapp_source is None:
                oapp_source = date_bar[0].find('a', {'node-type':'feed_list_item_date'}).find_next_sibling("a")
            oapp_source = oapp_source.text
            oapp_source = strip_blanks(oapp_source)
            return oapp_source
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_created_at(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            date = date_bar[-1]
            date = date.find('a', {'node-type':'feed_list_item_date'})
            created_at = date['title']
            return created_at
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ocreated_at(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            date = date_bar[0]
            date = date.find('a', {'node-type':'feed_list_item_date'})
            ocreated_at = date['title']
            return ocreated_at
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_favourite_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_feed_handle')
            favourite = options.find('span', {'node-type':'like_status'})
            favourite = favourite.find('em').text
            favourite = favourite.replace(' ', '')
            if favourite.isdigit():
                return int(favourite)
            else:
                return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ofavourite_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_handle W_fr')
            options = options.find_all('li')
            for opt in options:
                if opt.find('em') is not None:
                    ofavourite = opt.find('em').text
                    ofavourite = ofavourite.replace(' ', '')
                    if ofavourite.isdigit():
                        return int(ofavourite)
                    else:
                        return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_comment_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_feed_handle')
            comment = options.find('span', {'class':'line S_line1', 'node-type':'comment_btn_text'})
            comment = comment.text.strip(u'评论 ')
            comment = comment.replace(' ', '')
            if comment.isdigit():
                return int(comment)
            else:
                return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_ocomment_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_handle W_fr')
            options = options.find_all('li')
            for opt in options:
                if opt.text.find(u'评论') != -1:
                    ocomment = opt
                    break
            ocomment = ocomment.find('a', {'class':'S_txt2', 'bpfilter':'page_frame'})
            ocomment = ocomment.text.strip(u'评论 ')
            ocomment = ocomment.replace(' ', '')
            if ocomment.isdigit():
                return int(ocomment)
            else:
                return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_retweet_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_feed_handle')
            retweet = options.find('span', {'class':'line S_line1', 'node-type':'forward_btn_text'})
            retweet = retweet.text.strip(u'转发 ')
            retweet = retweet.replace(' ', '')
            if retweet.isdigit():
                return int(retweet)
            else:
                return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_oretweet_num(self, timeline):
        try:
            options = timeline.find('div', 'WB_handle W_fr')
            options = options.find_all('li')
            for opt in options:
                if opt.text.find(u'转发') != -1:
                    oretweet = opt
                    break
            oretweet = oretweet.find('a', {'class':'S_txt2', 'bpfilter':'page_frame'})
            oretweet = oretweet.text.strip(u'转发 ')
            oretweet = oretweet.replace(' ', '')
            if oretweet.isdigit():
                return int(oretweet)
            else:
                return 0
        except Exception as e:
            Logger.logger.debug(e)
            return None

    def parse_uid(self, timeline):
        try:
            uid = timeline['tbinfo'].split('&')[0]
            uid = uid.split('=')[1]
            return uid
        except Exception as e:
            Logger.logger.debug(e)
            raise e
    def parse_ouid(self, timeline):
        try:
            ouid = timeline['tbinfo'].split('&')[1]
            ouid = ouid.split('=')[1]
            return ouid
        except Exception as e:
            Logger.logger.debug(e)
            raise e

    def parse_mid(self, timeline):
        try:
            mid = timeline['mid']
            return mid
        except KeyError as ke:
            raise ke
    def parse_omid(self, timeline):
        try:
            omid = timeline['omid']
            return omid
        except KeyError as ke:
            raise ke

    def parse_encrypt_mid(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            date = date_bar[-1]
            date = date.find('a', {'node-type':'feed_list_item_date'})
            date = date['href'].split('/')[-1]
            date = date.split('?from=')[0]
            return date
        except Exception as e:
            Logger.logger.debug(e)
            return None
    def parse_encrypt_omid(self, timeline):
        date_bar = timeline.find_all('div', 'WB_from S_txt2')
        try:
            date = date_bar[0]
            date = date.find('a', {'node-type':'feed_list_item_date'})
            date = date['href'].split('/')[-1]
            date = date.split('?from=')[0]
            return date
        except Exception as e:
            Logger.logger.debug(e)
            return None
